# KERNEL ?= sgemm_x_cublas
# KERNEL ?= sgemm_0_naive
# KERNEL ?= sgemm_1_coalesced
# KERNEL ?= sgemm_1_shared_memory
# KERNEL ?= sgemm_2_joint_tiling
# KERNEL ?= sgemm_3_loop_interchange_64x128x16_8x16_f4
# KERNEL ?= sgemm_3_loop_interchange_128x128x16_16x16_f4
# KERNEL ?= sgemm_4_bank_conflict
KERNEL ?= sgemm_5_double_buffer

# ARCH ?= 70 75 80 86
ARCH ?= 80
NCU := ncu

CC         := nvcc 
LINKER     := $(CC)
#CFLAGS     := -O0 -g -Wall
$(foreach sm,$(ARCH),$(eval GENCODE_FLAGS += -gencode arch=compute_$(sm),code=sm_$(sm)))
CFLAGS     := -std=c++17 -O2 -g --resource-usage -lineinfo
CFLAGS += -w #设置警告级别为最低
LDFLAGS    := -lm  -lcublas

UTIL       := obj/compare_matrices.o \
              obj/random_matrix.o \
              obj/sgemm_cublas.o

TEST_OBJS  := obj/test_sgemm.o obj/$(KERNEL).o 

ncu_section ?= --section SpeedOfLight --section SpeedOfLight_HierarchicalTensorRooflineChart --section LaunchStats\
				 --section SpeedOfLight_RooflineChart --section Occupancy --section WarpStateStats --section SchedulerStats --section InstructionStats\
				 --section MemoryWorkloadAnalysis_Chart --section MemoryWorkloadAnalysis_Tables --section ComputeWorkloadAnalysis

ncu_metrics ?= sm__sass_data_bytes_mem_shared_op_ld.sum,sm__sass_inst_executed_op_shared_ld.sum,l1tex__data_pipe_lsu_wavefronts_mem_shared_op_ld.sum,l1tex__data_pipe_lsu_wavefronts_mem_shared_op_ld.sum.pct_of_peak_sustained_elapsed,l1tex__data_bank_conflicts_pipe_lsu_mem_shared_op_ld.sum,sm__sass_data_bytes_mem_shared_op_st.sum,sm__sass_inst_executed_op_shared_st.sum,l1tex__data_pipe_lsu_wavefronts_mem_shared_op_st.sum,l1tex__data_pipe_lsu_wavefronts_mem_shared_op_st.sum.pct_of_peak_sustained_elapsed,l1tex__data_bank_conflicts_pipe_lsu_mem_shared_op_st.sum

# define ncu_metrics_raw
# sm__sass_data_bytes_mem_shared_op_ld.sum
# sm__sass_inst_executed_op_shared_ld.sum
# l1tex__data_pipe_lsu_wavefronts_mem_shared_op_ld.sum
# l1tex__data_pipe_lsu_wavefronts_mem_shared_op_ld.sum.pct_of_peak_sustained_elapsed
# l1tex__data_bank_conflicts_pipe_lsu_mem_shared_op_ld.sum
# sm__sass_data_bytes_mem_shared_op_st.sum
# sm__sass_inst_executed_op_shared_st.sum
# l1tex__data_pipe_lsu_wavefronts_mem_shared_op_st.sum
# l1tex__data_pipe_lsu_wavefronts_mem_shared_op_st.sum.pct_of_peak_sustained_elapsed
# l1tex__data_bank_conflicts_pipe_lsu_mem_shared_op_st.sum
# endef

# ncu_metrics := $(shell echo $(ncu_metrics_raw) | tr '\n' ',' | sed 's/,$$//')


all: 
#	make clean;
	make bin/$(KERNEL).x

obj/%.o: %.cpp
	mkdir -p obj
	$(CC) $(CFLAGS) $(GENCODE_FLAGS)  -c $< -o $@

obj/%.o: %.cu
	mkdir -p obj
	$(CC) $(CFLAGS) $(GENCODE_FLAGS)  -c $< -o $@


bin/$(KERNEL).x: $(TEST_OBJS) $(UTIL) Makefile
	mkdir -p bin
	$(LINKER) $(TEST_OBJS) $(UTIL) $(LDFLAGS) \
        $(BLAS_LIB) -o $(TEST_BIN) $@ 

output:	all
	mkdir -p output
	make clean;
	make all
	./bin/$(KERNEL).x  > output/$(KERNEL).csv

plot:
	python3 plot.py output

clean:
	rm -f obj/*.o *~ bin/*.x 

cleanall:
	rm -f obj/*.o *~ bin/*.x output/*.csv output/*.png ncu/*.ncu-rep

run: bin/$(KERNEL).x
	./bin/$(KERNEL).x 

ncu_section: bin/$(KERNEL).x
	mkdir -p ncu
	$(NCU) -f -o ncu/$(KERNEL).ncu-rep   $(ncu_section) ./bin/$(KERNEL).x -t 

ncu_metrics: bin/$(KERNEL).x
	mkdir -p ncu
	$(NCU) --metrics $(ncu_metrics) ./bin/$(KERNEL).x -t